{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics\n",
    "\n",
    "> This contains metrics not included in fastai."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from tsai.imports import *\n",
    "from fastai.metrics import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "mk_class('ActivationType', **{o:o.lower() for o in ['No', 'Sigmoid', 'Softmax', 'BinarySoftmax']},\n",
    "         doc=\"All possible activation classes for `AccumMetric\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def MatthewsCorrCoefBinary(sample_weight=None):\n",
    "    \"Matthews correlation coefficient for single-label classification problems\"\n",
    "    return AccumMetric(skm.matthews_corrcoef, dim_argmax=-1, activation=ActivationType.BinarySoftmax, thresh=.5, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def get_task_metrics(dls, binary_metrics=None, multi_class_metrics=None, regression_metrics=None, verbose=True): \n",
    "    if dls.c == 2: \n",
    "        pv('binary-classification task', verbose)\n",
    "        return binary_metrics\n",
    "    elif dls.c > 2: \n",
    "        pv('multi-class task', verbose)\n",
    "        return multi_class_metrics\n",
    "    else:\n",
    "        pv('regression task', verbose)\n",
    "        return regression_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All metrics applicable to multi classification have been created by Doug Williams (https://github.com/williamsdoug). Thanks a lot Doug!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def accuracy_multi(inp, targ, thresh=0.5, sigmoid=True, by_sample=False):\n",
    "    \"Computes accuracy when `inp` and `targ` are the same size.\"\n",
    "    if sigmoid: inp = inp.sigmoid()\n",
    "    correct = (inp>thresh)==targ.bool()\n",
    "    if by_sample:\n",
    "        return (correct.float().mean(-1) == 1).float().mean()\n",
    "    else:\n",
    "        inp,targ = flatten_check(inp,targ)\n",
    "        return correct.float().mean()\n",
    "    \n",
    "def metrics_multi_common(inp, targ, thresh=0.5, sigmoid=True, by_sample=False):\n",
    "    \"Computes TP, TN, FP, FN when `inp` and `targ` are the same size.\"\n",
    "    if not by_sample: inp,targ = flatten_check(inp,targ)\n",
    "    if sigmoid: inp = inp.sigmoid()\n",
    "    pred = inp>thresh\n",
    "    \n",
    "    correct = pred==targ.bool()\n",
    "    TP = torch.logical_and(correct, (targ==1).bool()).sum()\n",
    "    TN = torch.logical_and(correct, (targ==0).bool()).sum()\n",
    "    \n",
    "    incorrect = pred!=targ.bool()\n",
    "    FN = torch.logical_and(incorrect, (targ==1).bool()).sum()\n",
    "    FP = torch.logical_and(incorrect, (targ==0).bool()).sum()\n",
    "    \n",
    "    N =  targ.size()[0]\n",
    "    return N, TP, TN, FP, FN\n",
    "\n",
    "def precision_multi(inp, targ, thresh=0.5, sigmoid=True):\n",
    "    \"Computes precision when `inp` and `targ` are the same size.\"\n",
    "    \n",
    "    inp,targ = flatten_check(inp,targ)\n",
    "    if sigmoid: inp = inp.sigmoid()\n",
    "    pred = inp>thresh\n",
    "    \n",
    "    correct = pred==targ.bool()\n",
    "    TP = torch.logical_and(correct,  (targ==1).bool()).sum()\n",
    "    FP = torch.logical_and(~correct, (targ==0).bool()).sum()\n",
    "\n",
    "    precision = TP/(TP+FP)\n",
    "    return precision\n",
    "\n",
    "def recall_multi(inp, targ, thresh=0.5, sigmoid=True):\n",
    "    \"Computes recall when `inp` and `targ` are the same size.\"\n",
    "    \n",
    "    inp,targ = flatten_check(inp,targ)\n",
    "    if sigmoid: inp = inp.sigmoid()\n",
    "    pred = inp>thresh\n",
    "    \n",
    "    correct = pred==targ.bool()\n",
    "    TP = torch.logical_and(correct,  (targ==1).bool()).sum()\n",
    "    FN = torch.logical_and(~correct, (targ==1).bool()).sum()\n",
    "\n",
    "    recall = TP/(TP+FN)\n",
    "    return recall\n",
    "\n",
    "def specificity_multi(inp, targ, thresh=0.5, sigmoid=True):\n",
    "    \"Computes specificity (true negative rate) when `inp` and `targ` are the same size.\"\n",
    "    \n",
    "    inp,targ = flatten_check(inp,targ)\n",
    "    if sigmoid: inp = inp.sigmoid()\n",
    "    pred = inp>thresh\n",
    "    \n",
    "    correct = pred==targ.bool()\n",
    "    TN = torch.logical_and(correct,  (targ==0).bool()).sum()\n",
    "    FP = torch.logical_and(~correct, (targ==0).bool()).sum()\n",
    "\n",
    "    specificity = TN/(TN+FP)\n",
    "    return specificity\n",
    "\n",
    "def balanced_accuracy_multi(inp, targ, thresh=0.5, sigmoid=True):\n",
    "    \"Computes balanced accuracy when `inp` and `targ` are the same size.\"\n",
    "    \n",
    "    inp,targ = flatten_check(inp,targ)\n",
    "    if sigmoid: inp = inp.sigmoid()\n",
    "    pred = inp>thresh\n",
    "    \n",
    "    correct = pred==targ.bool()\n",
    "    TP = torch.logical_and(correct,  (targ==1).bool()).sum()\n",
    "    TN = torch.logical_and(correct,  (targ==0).bool()).sum()\n",
    "    FN = torch.logical_and(~correct, (targ==1).bool()).sum()\n",
    "    FP = torch.logical_and(~correct, (targ==0).bool()).sum()\n",
    "\n",
    "    TPR = TP/(TP+FN)\n",
    "    TNR = TN/(TN+FP)\n",
    "    balanced_accuracy = (TPR+TNR)/2\n",
    "    return balanced_accuracy\n",
    "\n",
    "def Fbeta_multi(inp, targ, beta=1.0, thresh=0.5, sigmoid=True):\n",
    "    \"Computes Fbeta when `inp` and `targ` are the same size.\"\n",
    "    \n",
    "    inp,targ = flatten_check(inp,targ)\n",
    "    if sigmoid: inp = inp.sigmoid()\n",
    "    pred = inp>thresh\n",
    "    \n",
    "    correct = pred==targ.bool()\n",
    "    TP = torch.logical_and(correct,  (targ==1).bool()).sum()\n",
    "    TN = torch.logical_and(correct,  (targ==0).bool()).sum()\n",
    "    FN = torch.logical_and(~correct, (targ==1).bool()).sum()\n",
    "    FP = torch.logical_and(~correct, (targ==0).bool()).sum()\n",
    "\n",
    "    precision = TP/(TP+FP)\n",
    "    recall = TP/(TP+FN)\n",
    "    beta2 = beta*beta\n",
    "    \n",
    "    if precision+recall > 0:\n",
    "        Fbeta = (1+beta2)*precision*recall/(beta2*precision+recall)\n",
    "    else:\n",
    "        Fbeta = 0\n",
    "    return Fbeta\n",
    "\n",
    "def F1_multi(*args, **kwargs):\n",
    "    return Fbeta_multi(*args, **kwargs)  # beta defaults to 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "\n",
    "def mae(inp,targ):\n",
    "    \"Mean absolute error between `inp` and `targ`.\"\n",
    "    inp,targ = flatten_check(inp,targ)\n",
    "    return torch.abs(inp - targ).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "\n",
    "def mape(inp,targ):\n",
    "    \"Mean absolute percentage error between `inp` and `targ`.\"\n",
    "    inp,targ = flatten_check(inp, targ)\n",
    "    return (torch.abs(inp - targ) / torch.clamp_min(targ, 1e-8)).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "\n",
    "def _mean_per_class_accuracy(y_true, y_pred, *, labels=None, sample_weight=None, normalize=None):\n",
    "    cm = skm.confusion_matrix(y_true, y_pred, labels=labels, sample_weight=sample_weight, normalize=normalize)\n",
    "    return (cm.diagonal() / cm.sum(1)).mean()\n",
    "\n",
    "mean_per_class_accuracy = skm_to_fastai(_mean_per_class_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 000_export.ipynb.\n",
      "Converted 001_utils.ipynb.\n",
      "Converted 010_data.validation.ipynb.\n",
      "Converted 011_data.preparation.ipynb.\n",
      "Converted 012_data.external.ipynb.\n",
      "Converted 013_data.core.ipynb.\n",
      "Converted 014_data.unwindowed.ipynb.\n",
      "Converted 015_data.metadatasets.ipynb.\n",
      "Converted 016_data.preprocessing.ipynb.\n",
      "Converted 017_data.transforms.ipynb.\n",
      "Converted 018_data.mixed_augmentation.ipynb.\n",
      "Converted 019_data.image.ipynb.\n",
      "Converted 020_data.features.ipynb.\n",
      "Converted 021_data.tabular.ipynb.\n",
      "Converted 022_data.mixed.ipynb.\n",
      "Converted 050_losses.ipynb.\n",
      "Converted 051_metrics.ipynb.\n",
      "Converted 052_learner.ipynb.\n",
      "Converted 052b_tslearner.ipynb.\n",
      "Converted 053_optimizer.ipynb.\n",
      "Converted 060_callback.core.ipynb.\n",
      "Converted 061_callback.noisy_student.ipynb.\n",
      "Converted 063_callback.MVP.ipynb.\n",
      "Converted 064_callback.PredictionDynamics.ipynb.\n",
      "Converted 100_models.layers.ipynb.\n",
      "Converted 100b_models.utils.ipynb.\n",
      "Converted 100c_models.explainability.ipynb.\n",
      "Converted 101_models.ResNet.ipynb.\n",
      "Converted 101b_models.ResNetPlus.ipynb.\n",
      "Converted 102_models.InceptionTime.ipynb.\n",
      "Converted 102b_models.InceptionTimePlus.ipynb.\n",
      "Converted 103_models.MLP.ipynb.\n",
      "Converted 103b_models.FCN.ipynb.\n",
      "Converted 103c_models.FCNPlus.ipynb.\n",
      "Converted 104_models.ResCNN.ipynb.\n",
      "Converted 105_models.RNN.ipynb.\n",
      "Converted 105_models.RNNPlus.ipynb.\n",
      "Converted 106_models.XceptionTime.ipynb.\n",
      "Converted 106b_models.XceptionTimePlus.ipynb.\n",
      "Converted 107_models.RNN_FCN.ipynb.\n",
      "Converted 107b_models.RNN_FCNPlus.ipynb.\n",
      "Converted 108_models.TransformerModel.ipynb.\n",
      "Converted 108b_models.TST.ipynb.\n",
      "Converted 108c_models.TSTPlus.ipynb.\n",
      "Converted 109_models.OmniScaleCNN.ipynb.\n",
      "Converted 110_models.mWDN.ipynb.\n",
      "Converted 111_models.ROCKET.ipynb.\n",
      "Converted 111b_models.MINIROCKET.ipynb.\n",
      "Converted 111c_models.MINIROCKET_Pytorch.ipynb.\n",
      "Converted 111d_models.MINIROCKETPlus_Pytorch.ipynb.\n",
      "Converted 112_models.XResNet1d.ipynb.\n",
      "Converted 112b_models.XResNet1dPlus.ipynb.\n",
      "Converted 113_models.TCN.ipynb.\n",
      "Converted 114_models.XCM.ipynb.\n",
      "Converted 114b_models.XCMPlus.ipynb.\n",
      "Converted 120_models.TabModel.ipynb.\n",
      "Converted 121_models.TabTransformer.ipynb.\n",
      "Converted 122_models.TabFusionTransformer.ipynb.\n",
      "Converted 123_models.TSPerceiver.ipynb.\n",
      "Converted 124_models.TSiTPlus.ipynb.\n",
      "Converted 130_models.MultiInputNet.ipynb.\n",
      "Converted 140_models.misc.ipynb.\n",
      "Converted 200_optuna.ipynb.\n",
      "Converted 201_wandb.ipynb.\n",
      "Converted 900_tutorials.ipynb.\n",
      "Converted index.ipynb.\n",
      "\n",
      "\n",
      "Checking folder: /Users/nacho/notebooks/tsai/tsai\n",
      "Correct conversion! 😃\n",
      "Total time elapsed 337.160 s\n",
      "Tuesday 12/10/21 09:44:53 CEST\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#hide\n",
    "out = create_scripts(); beep(out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
